library(here)
library(tidyverse)
library(cowplot)
library(GGally)
theme_set(theme_light(base_size = 14))

library(scuttle)
library(scater)
library(scran)

library(kableExtra)
library(SCandwichCompanion)
library(iCOBRA)

Setup

## Directory setup
here_root <- "benchmarks/lupus"
here::i_am(file.path(here_root, "analysis/lupus-sim-results.Rmd"))
#> here() starts at /Users/milan/Projects/SCandwich-analyses

data_dir <- here::here(here_root, "data")
stopifnot(dir.exists(data_dir))
res_dir <- here::here(here_root, "results")
stopifnot(dir.exists(res_dir))
  • Using methods: muscat, sandwichTtestLRPrior1, pseudobulkSandwichTtest, NBpseudobulkSandwichTtest
  • Using celltype: combined
  • Using prop_DE: 0.05

Load results

res_files <- map(prop_DE, ~ get_sim_res_files(
    dataset = "lupus", methods = methods,
    prop_DE = .x, celltype = celltype
)) %>% set_names(paste0("prop_DE_", prop_DE))
res_list <- map_depth(res_files, 2, readRDS)

## Check for expected data structure
stopifnot(unlist(map_depth(res_list, 4, ~ is.data.frame(.x$results))))

Load SCE objects

sce_files <- map(prop_DE, ~ get_SCE_files(
    dataset = "lupus",
    which = "sim_replicates", prop_DE = .x,
    celltype = celltype
)) %>% set_names(paste0("prop_DE_", prop_DE))
sce_objects <- lapply(sce_files, readRDS)

Data overview

  • The results were generated from 1 simulation replicates
  • Each replicate was generated by randomly splitting the subjects in two mock groups
  • The same mock replicates were used for each prop_DE level. I.e. prop_DE_0.1$replicate1 and prop_DE_0.05$replicate1 will have the same mock group assignment
  • DE was introduced by swapping genes in one of the mock groups
  • Cells per subject were sub-sampled to 100 cells per subject
map_dfr(sce_objects,
    ~ map_dfr(.x, function(x) c(nrows = nrow(x), ncols = ncol(x)),
        .id = "replicate"
    ),
    .id = "prop_DE"
)

Subjects are divided across mock groups as follows:

lapply(sce_objects[[1]], function(x) table(x$ind_cov, x$mock_group))
#> $replicate_1
#>                    
#>                        A    B
#>   IGTB141_IGTB141    964    0
#>   IGTB143_IGTB143   1417    0
#>   IGTB195_IGTB195      0 1571
#>   IGTB469_IGTB469   1263    0
#>   IGTB498_IGTB498   1594    0
#>   IGTB508_IGTB508      0 1317
#>   IGTB514_IGTB514   1333    0
#>   IGTB645_IGTB645      0 1299
#>   IGTB670_IGTB670   1290    0
#>   IGTB826_IGTB826      0 1154
#>   IGTB884_IGTB884      0  967
#>   IGTB986_IGTB986    926    0
#>   IGTB1372_IGTB1372    0  836
#>   IGTB1506_IGTB1506    0  927
#>   IGTB1539_IGTB1539    0  818
#>   IGTB1575_IGTB1575  823    0
#>   IGTB1650_IGTB1650 1238    0
#>   IGTB1762_IGTB1762 1815    0
#>   IGTB1768_IGTB1768 1545    0
#>   IGTB1789_IGTB1789    0 1105
#>   IGTB1793_IGTB1793 1233    0
#>   IGTB1814_IGTB1814    0 1400
#>   IGTB1815_IGTB1815 1063    0
#>   IGTB1826_IGTB1826    0  974
#>   IGTB1827_IGTB1827 1117    0
#>   IGTB1828_IGTB1828  959    0
#>   IGTB1840_IGTB1840 1422    0
#>   IGTB1844_IGTB1844 1252    0
#>   IGTB1871_IGTB1871    0  973
#>   IGTB1895_IGTB1895    0 1098
#>   IGTB1901_IGTB1901    0  960
#>   IGTB1906_IGTB1906  779    0
#>   IGTB1908_IGTB1908    0  773
#>   IGTB1913_IGTB1913    0 1659
#>   IGTB1916_IGTB1916    0  985
#>   IGTB1921_IGTB1921 1468    0
#>   IGTB1952_IGTB1952    0 1263
#>   IGTB1966_IGTB1966    0 1782
#>   IGTB1974_IGTB1974  957    0
#>   IGTB1982_IGTB1982    0  827
#>   IGTB1987_IGTB1987    0 1526
#>   IGTB1996_IGTB1996    0  622
#>   IGTB2007_IGTB2007 1001    0
#>   IGTB2065_IGTB2065  954    0
#> 
#> $replicate_2
#>                    
#>                        A    B
#>   IGTB141_IGTB141      0  964
#>   IGTB143_IGTB143   1417    0
#>   IGTB195_IGTB195   1571    0
#>   IGTB469_IGTB469      0 1263
#>   IGTB498_IGTB498   1594    0
#>   IGTB508_IGTB508   1317    0
#>   IGTB514_IGTB514      0 1333
#>   IGTB645_IGTB645      0 1299
#>   IGTB670_IGTB670      0 1290
#>   IGTB826_IGTB826   1154    0
#>   IGTB884_IGTB884    967    0
#>   IGTB986_IGTB986      0  926
#>   IGTB1372_IGTB1372    0  836
#>   IGTB1506_IGTB1506    0  927
#>   IGTB1539_IGTB1539  818    0
#>   IGTB1575_IGTB1575    0  823
#>   IGTB1650_IGTB1650    0 1238
#>   IGTB1762_IGTB1762 1815    0
#>   IGTB1768_IGTB1768 1545    0
#>   IGTB1789_IGTB1789    0 1105
#>   IGTB1793_IGTB1793    0 1233
#>   IGTB1814_IGTB1814    0 1400
#>   IGTB1815_IGTB1815 1063    0
#>   IGTB1826_IGTB1826  974    0
#>   IGTB1827_IGTB1827 1117    0
#>   IGTB1828_IGTB1828    0  959
#>   IGTB1840_IGTB1840 1422    0
#>   IGTB1844_IGTB1844 1252    0
#>   IGTB1871_IGTB1871    0  973
#>   IGTB1895_IGTB1895 1098    0
#>   IGTB1901_IGTB1901  960    0
#>   IGTB1906_IGTB1906  779    0
#>   IGTB1908_IGTB1908  773    0
#>   IGTB1913_IGTB1913    0 1659
#>   IGTB1916_IGTB1916    0  985
#>   IGTB1921_IGTB1921 1468    0
#>   IGTB1952_IGTB1952    0 1263
#>   IGTB1966_IGTB1966 1782    0
#>   IGTB1974_IGTB1974    0  957
#>   IGTB1982_IGTB1982  827    0
#>   IGTB1987_IGTB1987    0 1526
#>   IGTB1996_IGTB1996    0  622
#>   IGTB2007_IGTB2007    0 1001
#>   IGTB2065_IGTB2065  954    0
#> 
#> $replicate_3
#>                    
#>                        A    B
#>   IGTB141_IGTB141      0  964
#>   IGTB143_IGTB143      0 1417
#>   IGTB195_IGTB195   1571    0
#>   IGTB469_IGTB469      0 1263
#>   IGTB498_IGTB498   1594    0
#>   IGTB508_IGTB508   1317    0
#>   IGTB514_IGTB514      0 1333
#>   IGTB645_IGTB645   1299    0
#>   IGTB670_IGTB670      0 1290
#>   IGTB826_IGTB826   1154    0
#>   IGTB884_IGTB884    967    0
#>   IGTB986_IGTB986    926    0
#>   IGTB1372_IGTB1372  836    0
#>   IGTB1506_IGTB1506  927    0
#>   IGTB1539_IGTB1539    0  818
#>   IGTB1575_IGTB1575    0  823
#>   IGTB1650_IGTB1650    0 1238
#>   IGTB1762_IGTB1762    0 1815
#>   IGTB1768_IGTB1768 1545    0
#>   IGTB1789_IGTB1789 1105    0
#>   IGTB1793_IGTB1793 1233    0
#>   IGTB1814_IGTB1814    0 1400
#>   IGTB1815_IGTB1815 1063    0
#>   IGTB1826_IGTB1826    0  974
#>   IGTB1827_IGTB1827    0 1117
#>   IGTB1828_IGTB1828    0  959
#>   IGTB1840_IGTB1840 1422    0
#>   IGTB1844_IGTB1844    0 1252
#>   IGTB1871_IGTB1871  973    0
#>   IGTB1895_IGTB1895    0 1098
#>   IGTB1901_IGTB1901    0  960
#>   IGTB1906_IGTB1906  779    0
#>   IGTB1908_IGTB1908    0  773
#>   IGTB1913_IGTB1913 1659    0
#>   IGTB1916_IGTB1916  985    0
#>   IGTB1921_IGTB1921 1468    0
#>   IGTB1952_IGTB1952 1263    0
#>   IGTB1966_IGTB1966    0 1782
#>   IGTB1974_IGTB1974  957    0
#>   IGTB1982_IGTB1982    0  827
#>   IGTB1987_IGTB1987    0 1526
#>   IGTB1996_IGTB1996  622    0
#>   IGTB2007_IGTB2007    0 1001
#>   IGTB2065_IGTB2065  954    0
#> 
#> $replicate_4
#>                    
#>                        A    B
#>   IGTB141_IGTB141      0  964
#>   IGTB143_IGTB143      0 1417
#>   IGTB195_IGTB195      0 1571
#>   IGTB469_IGTB469   1263    0
#>   IGTB498_IGTB498      0 1594
#>   IGTB508_IGTB508   1317    0
#>   IGTB514_IGTB514   1333    0
#>   IGTB645_IGTB645      0 1299
#>   IGTB670_IGTB670      0 1290
#>   IGTB826_IGTB826   1154    0
#>   IGTB884_IGTB884    967    0
#>   IGTB986_IGTB986    926    0
#>   IGTB1372_IGTB1372  836    0
#>   IGTB1506_IGTB1506    0  927
#>   IGTB1539_IGTB1539  818    0
#>   IGTB1575_IGTB1575    0  823
#>   IGTB1650_IGTB1650 1238    0
#>   IGTB1762_IGTB1762    0 1815
#>   IGTB1768_IGTB1768    0 1545
#>   IGTB1789_IGTB1789    0 1105
#>   IGTB1793_IGTB1793    0 1233
#>   IGTB1814_IGTB1814 1400    0
#>   IGTB1815_IGTB1815 1063    0
#>   IGTB1826_IGTB1826  974    0
#>   IGTB1827_IGTB1827    0 1117
#>   IGTB1828_IGTB1828    0  959
#>   IGTB1840_IGTB1840 1422    0
#>   IGTB1844_IGTB1844 1252    0
#>   IGTB1871_IGTB1871  973    0
#>   IGTB1895_IGTB1895 1098    0
#>   IGTB1901_IGTB1901    0  960
#>   IGTB1906_IGTB1906  779    0
#>   IGTB1908_IGTB1908    0  773
#>   IGTB1913_IGTB1913    0 1659
#>   IGTB1916_IGTB1916    0  985
#>   IGTB1921_IGTB1921    0 1468
#>   IGTB1952_IGTB1952    0 1263
#>   IGTB1966_IGTB1966 1782    0
#>   IGTB1974_IGTB1974  957    0
#>   IGTB1982_IGTB1982    0  827
#>   IGTB1987_IGTB1987    0 1526
#>   IGTB1996_IGTB1996  622    0
#>   IGTB2007_IGTB2007 1001    0
#>   IGTB2065_IGTB2065    0  954
#> 
#> $replicate_5
#>                    
#>                        A    B
#>   IGTB141_IGTB141      0  964
#>   IGTB143_IGTB143      0 1417
#>   IGTB195_IGTB195   1571    0
#>   IGTB469_IGTB469      0 1263
#>   IGTB498_IGTB498   1594    0
#>   IGTB508_IGTB508      0 1317
#>   IGTB514_IGTB514   1333    0
#>   IGTB645_IGTB645   1299    0
#>   IGTB670_IGTB670      0 1290
#>   IGTB826_IGTB826   1154    0
#>   IGTB884_IGTB884    967    0
#>   IGTB986_IGTB986      0  926
#>   IGTB1372_IGTB1372  836    0
#>   IGTB1506_IGTB1506    0  927
#>   IGTB1539_IGTB1539    0  818
#>   IGTB1575_IGTB1575  823    0
#>   IGTB1650_IGTB1650    0 1238
#>   IGTB1762_IGTB1762    0 1815
#>   IGTB1768_IGTB1768    0 1545
#>   IGTB1789_IGTB1789    0 1105
#>   IGTB1793_IGTB1793 1233    0
#>   IGTB1814_IGTB1814 1400    0
#>   IGTB1815_IGTB1815 1063    0
#>   IGTB1826_IGTB1826  974    0
#>   IGTB1827_IGTB1827 1117    0
#>   IGTB1828_IGTB1828  959    0
#>   IGTB1840_IGTB1840 1422    0
#>   IGTB1844_IGTB1844    0 1252
#>   IGTB1871_IGTB1871  973    0
#>   IGTB1895_IGTB1895 1098    0
#>   IGTB1901_IGTB1901    0  960
#>   IGTB1906_IGTB1906  779    0
#>   IGTB1908_IGTB1908  773    0
#>   IGTB1913_IGTB1913 1659    0
#>   IGTB1916_IGTB1916    0  985
#>   IGTB1921_IGTB1921    0 1468
#>   IGTB1952_IGTB1952    0 1263
#>   IGTB1966_IGTB1966    0 1782
#>   IGTB1974_IGTB1974    0  957
#>   IGTB1982_IGTB1982    0  827
#>   IGTB1987_IGTB1987 1526    0
#>   IGTB1996_IGTB1996  622    0
#>   IGTB2007_IGTB2007    0 1001
#>   IGTB2065_IGTB2065    0  954

The number of DE and non-DE genes per replicate:

map(sce_objects, ~ map_dfr(.x, ~ table(rowData(.x)$is_DE), .id = "replicate"))
#> $prop_DE_0.05
#> # A tibble: 5 × 3
#>   replicate   `FALSE` `TRUE` 
#>   <chr>       <table> <table>
#> 1 replicate_1 4283    225    
#> 2 replicate_2 4283    225    
#> 3 replicate_3 4283    225    
#> 4 replicate_4 4283    225    
#> 5 replicate_5 4284    224

t-SNE plots

t-SNE plots colored by celltype, shaped by mock_group.

tSNE_plots <- map(sce_objects, function(x) {
    p <- imap(x, function(sce, name) {
        plotTSNE(sce, colour_by = "ct_cov", shape_by = "mock_group") +
            ggtitle(name)
    })
    patchwork::wrap_plots(p, ncol = 3, guides = "collect")
})

Prop. DE = 0.05

Extract results of interest

Runtimes

## Nest methods below celltypes
res_list <- map(res_list, transpose)

## Get runtimes for each celltype
runtimes <- map_depth(res_list, 2,
    ~ map_dfr(.x, get_runtimes, depth = 1, .id = "method")
) %>%
    map(bind_rows, .id = "celltype")

P-values

res_tables <- map_depth(res_list, 3, get_aggregated_rep_tables, depth = 1)
res_tables <- map_depth(res_tables, 2, ~ combine_tables(.x, .id = "method"))

Visualize results

Run times

runtime_plots <- imap(
    runtimes,
    ~ plot_run_times(.x, width = 0.2, height = 0) +
        ggtitle(.y) +
        facet_wrap(vars(celltype))
)

prop_DE_0.05

P-value distributions

pval_figs <- map_depth(res_tables, 2, ~ pval_hist(.x))

prop_DE = 0.05

B_mem

#> Warning: Removed 455 rows containing non-finite values (stat_bin).

ncM

#> Warning: Removed 455 rows containing non-finite values (stat_bin).

T4_naive

#> Warning: Removed 455 rows containing non-finite values (stat_bin).

P-value distributions for non-DE genes

non_de_res <- map2(res_tables, sce_objects, function(res_by_ct, sce_by_rep) {
    map(res_by_ct, function(res_table) {
        res_by_rep <- split(res_table, res_table$replicate)
        out <- map2(res_by_rep, sce_by_rep, function(tbl, sce) {
            ## Select only non-DE genes
            non_de <- rownames(sce)[!rowData(sce)$is_DE]
            tbl[tbl$gene %in% non_de, ]
        })
        bind_rows(out, .id = "replicate")
    })
})
non_de_pval_figs <- map_depth(non_de_res, 2, ~ pval_hist(.x))

prop_DE = 0.05

B_mem

#> Warning: Removed 455 rows containing non-finite values (stat_bin).

ncM

#> Warning: Removed 455 rows containing non-finite values (stat_bin).

T4_naive

#> Warning: Removed 455 rows containing non-finite values (stat_bin).

Performance evaluation with iCOBRA

Prepare Data

P-values for missing genes are set to 1.

cobra_data <- map2(res_tables, sce_objects, function(res_by_ct, sce_by_rep) {
    map(res_by_ct, function(res_table) {
        res_by_rep <- split(res_table, res_table$replicate)
        map2(res_by_rep, sce_by_rep, prepare_COBRAData, replace_missing = TRUE)
    })
})

cobra_perf <- map_depth(cobra_data, 3, calculate_performance, binary_truth = "status")

n_methods <- length(unique(res_tables[[1]][[1]]$method)) + 1 # +1 for "truth"
cobra_objects <- map_depth(cobra_perf, 3, prepare_data_for_plot)

TPR plots

no_legend <- theme(legend.position = "none")
tpr_plots <- map_depth(cobra_objects, 2, function(cobra_list) {
    p <- imap(cobra_list, ~ plot_tpr(.x, title = .y) + no_legend)
    patchwork::wrap_plots(p, ncol = 3)
})

prop_DE = 0.05

B_mem

ncM

T4_naive

FPR plots

fpr_plots <- map_depth(cobra_objects, 2, function(cobra_list) {
    p <- imap(cobra_list, ~ plot_fpr(.x, title = .y) + no_legend)
    patchwork::wrap_plots(p, ncol = 3)
})

prop_DE = 0.05

B_mem

ncM

T4_naive

FDR-TPR curves

fdr_tpr_plots <- map_depth(cobra_objects, 2, function(cobra_list) {
    p <- imap(cobra_list, ~ plot_fdrtprcurve(.x, title = .y))
    patchwork::wrap_plots(p, ncol = 3, guides = "collect")
})

prop_DE = 0.05

B_mem

ncM

T4_naive

Zoomed FDR-TPR curves

Same plots but zoomed on the FDR-TPR points region.

fdr_tpr_plots_zoomed <- map_depth(cobra_objects, 2, function(cobra_list) {
    p <- imap(cobra_list, ~ plot_zoomed_fdrtprcurve(.x, title = .y))
    patchwork::wrap_plots(p, ncol = 3, guides = "collect")
})

prop_DE = 0.05

B_mem

ncM

T4_naive

Session info

Session info
#> [1] "2022-05-30 17:25:15 CEST"
#> Local:    lupus-data /Users/milan/Projects/SCandwich-analyses
#> Remote:   lupus-data @ origin (git@github.com:statOmics/SCandwich-analyses.git)
#> Head:     [60b481a] 2022-05-30: Small bug fix in sim-results reports
#> ─ Session info  đŸŒŠī¸  đŸ•ī¸  🏭   ─────────────────────────────────────────────────────────────────────────────────────────────────────
#>  hash: cloud with lightning, camping, factory
#> 
#>  setting  value
#>  version  R version 4.1.3 (2022-03-10)
#>  os       macOS Big Sur/Monterey 10.16
#>  system   x86_64, darwin17.0
#>  ui       X11
#>  language en_US.UTF-8
#>  collate  en_US.UTF-8
#>  ctype    en_US.UTF-8
#>  tz       Europe/Brussels
#>  date     2022-05-30
#>  pandoc   2.17.1.1 @ /usr/local/bin/ (via rmarkdown)
#> 
#> ─ Packages ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
#>  !  package              * version    date (UTC) lib source
#>  P  abind                  1.4-5      2016-07-21 [?] CRAN (R 4.1.0)
#>     annotate               1.72.0     2021-10-26 [1] Bioconductor
#>     AnnotationDbi          1.56.2     2021-11-09 [1] Bioconductor
#>     argparse             * 2.1.2      2021-10-21 [1] CRAN (R 4.1.0)
#>  P  assertthat             0.2.1      2019-03-21 [?] CRAN (R 4.1.0)
#>     backports              1.3.0      2021-10-27 [1] CRAN (R 4.1.0)
#>     beachmat               2.10.0     2021-10-26 [1] Bioconductor
#>  P  beeswarm               0.4.0      2021-06-01 [?] CRAN (R 4.1.0)
#>     Biobase              * 2.54.0     2021-10-26 [1] Bioconductor
#>     BiocGenerics         * 0.40.0     2021-10-26 [1] Bioconductor
#>  P  BiocManager            1.30.16    2021-06-15 [?] CRAN (R 4.1.0)
#>     BiocNeighbors          1.12.0     2021-10-26 [1] Bioconductor
#>     BiocParallel           1.28.1     2021-11-18 [1] Bioconductor
#>     BiocSingular           1.10.0     2021-10-26 [1] Bioconductor
#>     Biostrings             2.62.0     2021-10-26 [1] Bioconductor
#>  P  bit                    4.0.4      2020-08-04 [?] CRAN (R 4.1.0)
#>  P  bit64                  4.0.5      2020-08-30 [?] CRAN (R 4.1.0)
#>  P  bitops                 1.0-7      2021-04-24 [?] CRAN (R 4.1.0)
#>  P  blme                   1.0-5      2021-01-05 [?] CRAN (R 4.1.0)
#>  P  blob                   1.2.2      2021-07-23 [?] CRAN (R 4.1.0)
#>     bluster                1.4.0      2021-10-26 [1] Bioconductor
#>  P  boot                   1.3-28     2021-05-03 [?] CRAN (R 4.1.0)
#>     broom                  0.7.10     2021-10-31 [1] CRAN (R 4.1.0)
#>     bslib                  0.3.1      2021-10-06 [1] CRAN (R 4.1.0)
#>  P  cachem                 1.0.6      2021-08-19 [?] CRAN (R 4.1.0)
#>  P  caTools                1.18.2     2021-03-28 [?] CRAN (R 4.1.0)
#>  P  cellranger             1.1.0      2016-07-27 [?] CRAN (R 4.1.0)
#>  P  circlize               0.4.13     2021-06-09 [?] CRAN (R 4.1.0)
#>     cli                    3.1.0      2021-10-27 [1] CRAN (R 4.1.0)
#>     clue                   0.3-60     2021-10-11 [1] CRAN (R 4.1.0)
#>  P  cluster                2.1.2      2021-04-17 [?] CRAN (R 4.1.1)
#>  P  codetools              0.2-18     2020-11-04 [?] CRAN (R 4.1.0)
#>  P  colorspace             2.0-2      2021-06-24 [?] CRAN (R 4.1.0)
#>     ComplexHeatmap         2.10.0     2021-10-26 [1] Bioconductor
#>  P  cowplot              * 1.1.1      2020-12-30 [?] CRAN (R 4.1.0)
#>     crayon                 1.4.2      2021-10-29 [1] CRAN (R 4.1.0)
#>     data.table             1.14.2     2021-09-27 [1] CRAN (R 4.1.0)
#>  P  DBI                    1.1.1      2021-01-15 [?] CRAN (R 4.1.0)
#>  P  dbplyr                 2.1.1      2021-04-06 [?] CRAN (R 4.1.0)
#>     DelayedArray           0.20.0     2021-10-26 [1] Bioconductor
#>     DelayedMatrixStats     1.16.0     2021-10-26 [1] Bioconductor
#>     DESeq2                 1.34.0     2021-10-26 [1] Bioconductor
#>     digest                 0.6.28     2021-09-23 [1] CRAN (R 4.1.0)
#>  P  doParallel             1.0.16     2020-10-16 [?] CRAN (R 4.1.0)
#>  P  dplyr                * 1.0.7      2021-06-18 [?] CRAN (R 4.1.0)
#>  P  dqrng                  0.3.0      2021-05-01 [?] CRAN (R 4.1.0)
#>     DT                     0.20       2021-11-15 [1] CRAN (R 4.1.0)
#>     edgeR                  3.36.0     2021-10-26 [1] Bioconductor
#>  P  ellipsis               0.3.2      2021-04-29 [?] CRAN (R 4.1.0)
#>  P  evaluate               0.14       2019-05-28 [?] CRAN (R 4.1.0)
#>     fansi                  1.0.2      2022-01-14 [1] CRAN (R 4.1.2)
#>  P  farver                 2.1.0      2021-02-28 [?] CRAN (R 4.1.0)
#>  P  fastmap                1.1.0      2021-01-25 [?] CRAN (R 4.1.0)
#>  P  findpython             1.0.7      2021-01-27 [?] CRAN (R 4.1.0)
#>  P  forcats              * 0.5.1      2021-01-27 [?] CRAN (R 4.1.0)
#>  P  foreach                1.5.1      2020-10-15 [?] CRAN (R 4.1.0)
#>  P  fs                     1.5.0      2020-07-31 [?] CRAN (R 4.1.0)
#>     future                 1.23.0     2021-10-31 [1] CRAN (R 4.1.0)
#>  P  future.apply           1.8.1      2021-08-10 [?] CRAN (R 4.1.0)
#>  P  GEEutils               0.1.0      2022-04-28 [?] Github (jgilis/GEEutils@dd73301)
#>     genefilter             1.76.0     2021-10-26 [1] Bioconductor
#>     geneplotter            1.72.0     2021-10-26 [1] Bioconductor
#>     generics               0.1.1      2021-10-25 [1] CRAN (R 4.1.0)
#>     GenomeInfoDb         * 1.30.0     2021-10-26 [1] Bioconductor
#>     GenomeInfoDbData       1.2.7      2021-10-01 [1] Bioconductor
#>     GenomicRanges        * 1.46.1     2021-11-18 [1] Bioconductor
#>  P  GetoptLong             1.0.5      2020-12-15 [?] CRAN (R 4.1.0)
#>  P  GGally               * 2.1.2      2021-06-21 [?] CRAN (R 4.1.0)
#>  P  ggbeeswarm             0.6.0      2017-08-07 [?] CRAN (R 4.1.0)
#>     ggplot2              * 3.3.5.9000 2021-11-22 [1] Github (tidyverse/ggplot2@199eb90)
#>  P  ggrepel                0.9.1      2021-01-15 [?] CRAN (R 4.1.0)
#>  P  git2r                  0.29.0     2021-11-22 [?] CRAN (R 4.1.0)
#>  P  glmmTMB                1.1.2.3    2021-09-20 [?] CRAN (R 4.1.1)
#>  P  GlobalOptions          0.1.2      2020-06-10 [?] CRAN (R 4.1.0)
#>  P  globals                0.14.0     2020-11-22 [?] CRAN (R 4.1.0)
#>     glue                   1.6.0      2021-12-17 [1] CRAN (R 4.1.0)
#>  P  gplots                 3.1.1      2020-11-28 [?] CRAN (R 4.1.0)
#>  P  gridExtra              2.3        2017-09-09 [?] CRAN (R 4.1.0)
#>  P  gtable                 0.3.0      2019-03-25 [?] CRAN (R 4.1.0)
#>  P  gtools                 3.9.2      2021-06-06 [?] CRAN (R 4.1.0)
#>     haven                  2.4.3      2021-08-04 [1] CRAN (R 4.1.0)
#>  P  here                 * 1.0.1      2020-12-13 [?] CRAN (R 4.1.0)
#>  P  highr                  0.9        2021-04-16 [?] CRAN (R 4.1.0)
#>     hms                    1.1.1      2021-09-26 [1] CRAN (R 4.1.0)
#>  P  htmltools              0.5.2      2021-08-25 [?] CRAN (R 4.1.0)
#>     htmlwidgets            1.5.4      2021-09-08 [1] CRAN (R 4.1.0)
#>     httpuv                 1.6.3      2021-09-09 [1] CRAN (R 4.1.0)
#>  P  httr                   1.4.2      2020-07-20 [?] CRAN (R 4.1.0)
#>     iCOBRA               * 1.22.1     2021-11-03 [1] Bioconductor
#>     igraph                 1.2.8      2021-11-07 [1] CRAN (R 4.1.0)
#>     IRanges              * 2.28.0     2021-10-26 [1] Bioconductor
#>  P  irlba                  2.3.3      2019-02-05 [?] CRAN (R 4.1.0)
#>  P  iterators              1.0.13     2020-10-15 [?] CRAN (R 4.1.0)
#>  P  jquerylib              0.1.4      2021-04-26 [?] CRAN (R 4.1.0)
#>  P  jsonlite               1.7.2      2020-12-09 [?] CRAN (R 4.1.0)
#>  P  kableExtra           * 1.3.4      2021-02-20 [?] CRAN (R 4.1.0)
#>     KEGGREST               1.34.0     2021-10-26 [1] Bioconductor
#>  P  KernSmooth             2.23-20    2021-05-03 [?] CRAN (R 4.1.1)
#>     knitr                  1.36       2021-09-29 [1] CRAN (R 4.1.1)
#>  P  labeling               0.4.2      2020-10-20 [?] CRAN (R 4.1.0)
#>     later                  1.3.0      2021-08-18 [1] CRAN (R 4.1.0)
#>     lattice                0.20-45    2021-09-22 [1] CRAN (R 4.1.0)
#>     lifecycle              1.0.1      2021-09-24 [1] CRAN (R 4.1.0)
#>     limma                  3.50.0     2021-10-26 [1] Bioconductor
#>  P  listenv                0.8.0      2019-12-05 [?] CRAN (R 4.1.0)
#>  P  lme4                   1.1-27.1   2021-06-22 [?] CRAN (R 4.1.0)
#>  P  lmerTest               3.1-3      2020-10-23 [?] CRAN (R 4.1.0)
#>  P  locfit                 1.5-9.4    2020-03-25 [?] CRAN (R 4.1.0)
#>     lubridate              1.8.0      2021-10-07 [1] CRAN (R 4.1.0)
#>  P  magrittr               2.0.1      2020-11-17 [?] CRAN (R 4.1.0)
#>  P  MASS                   7.3-54     2021-05-03 [?] CRAN (R 4.1.0)
#>     MAST                   1.20.0     2021-10-26 [1] Bioconductor
#>  P  Matrix                 1.3-4      2021-06-01 [?] CRAN (R 4.1.0)
#>     MatrixGenerics       * 1.6.0      2021-10-26 [1] Bioconductor
#>     matrixStats          * 0.61.0     2021-09-17 [1] CRAN (R 4.1.0)
#>  P  memoise                2.0.0      2021-01-26 [?] CRAN (R 4.1.0)
#>     metapod                1.2.0      2021-10-26 [1] Bioconductor
#>     mime                   0.12       2021-09-28 [1] CRAN (R 4.1.0)
#>  P  minqa                  1.2.4      2014-10-09 [?] CRAN (R 4.1.0)
#>  P  modelr                 0.1.8      2020-05-19 [?] CRAN (R 4.1.0)
#>  P  munsell                0.5.0      2018-06-12 [?] CRAN (R 4.1.0)
#>     muscat                 1.8.0      2021-10-26 [1] Bioconductor
#>     nlme                   3.1-153    2021-09-07 [1] CRAN (R 4.1.0)
#>     nloptr                 1.2.2.3    2021-11-02 [1] CRAN (R 4.1.0)
#>  P  numDeriv               2016.8-1.1 2019-06-06 [?] CRAN (R 4.1.0)
#>     parallelly             1.29.0     2021-11-21 [1] CRAN (R 4.1.1)
#>  P  patchwork              1.1.1      2020-12-17 [?] CRAN (R 4.1.0)
#>  P  pbkrtest               0.5.1      2021-03-09 [?] CRAN (R 4.1.0)
#>     pillar                 1.6.4      2021-10-18 [1] CRAN (R 4.1.0)
#>  P  pkgconfig              2.0.3      2019-09-22 [?] CRAN (R 4.1.0)
#>  P  plyr                   1.8.6      2020-03-03 [?] CRAN (R 4.1.0)
#>  P  png                    0.1-7      2013-12-03 [?] CRAN (R 4.1.0)
#>  P  prettyunits            1.1.1      2020-01-24 [?] CRAN (R 4.1.0)
#>  P  progress               1.2.2      2019-05-16 [?] CRAN (R 4.1.0)
#>  P  promises               1.2.0.1    2021-02-11 [?] CRAN (R 4.1.0)
#>  P  purrr                * 0.3.4      2020-04-17 [?] CRAN (R 4.1.0)
#>  P  R6                     2.5.1      2021-08-19 [?] CRAN (R 4.1.0)
#>  P  RColorBrewer           1.1-2      2014-12-07 [?] CRAN (R 4.1.0)
#>  P  Rcpp                   1.0.7      2021-07-07 [?] CRAN (R 4.1.0)
#>     RCurl                  1.98-1.5   2021-09-17 [1] CRAN (R 4.1.0)
#>     readr                * 2.1.0      2021-11-11 [1] CRAN (R 4.1.0)
#>  P  readxl                 1.3.1      2019-03-13 [?] CRAN (R 4.1.0)
#>  P  renv                   0.15.4     2022-03-03 [?] CRAN (R 4.1.2)
#>  P  reprex                 2.0.1      2021-08-05 [?] CRAN (R 4.1.0)
#>  P  reshape                0.8.8      2018-10-23 [?] CRAN (R 4.1.0)
#>  P  reshape2               1.4.4      2020-04-09 [?] CRAN (R 4.1.0)
#>  P  rjson                  0.2.20     2018-06-08 [?] CRAN (R 4.1.0)
#>     rlang                  0.4.12     2021-10-18 [1] CRAN (R 4.1.0)
#>     rmarkdown              2.11       2021-09-14 [1] CRAN (R 4.1.0)
#>  P  ROCR                   1.0-11     2020-05-02 [?] CRAN (R 4.1.0)
#>  VP rprojroot              2.0.3      2020-11-15 [?] CRAN (R 4.1.0) (on disk 2.0.2)
#>  P  RSQLite                2.2.8      2021-08-21 [?] CRAN (R 4.1.0)
#>  P  rstudioapi             0.13       2020-11-12 [?] CRAN (R 4.1.0)
#>  P  rsvd                   1.0.5      2021-04-16 [?] CRAN (R 4.1.0)
#>     rvest                  1.0.2      2021-10-16 [1] CRAN (R 4.1.0)
#>     S4Vectors            * 0.32.2     2021-11-07 [1] Bioconductor
#>  P  sandwich               3.0-1      2021-05-18 [?] CRAN (R 4.1.0)
#>  P  sass                   0.4.0      2021-05-12 [?] CRAN (R 4.1.0)
#>     ScaledMatrix           1.2.0      2021-10-26 [1] Bioconductor
#>  P  scales                 1.1.1      2020-05-11 [?] CRAN (R 4.1.0)
#>     SCandwichCompanion   * 0.1.0      2022-05-04 [1] local
#>     scater               * 1.22.0     2021-10-26 [1] Bioconductor
#>     scran                * 1.22.1     2021-11-14 [1] Bioconductor
#>  P  sctransform            0.3.2      2020-12-16 [?] CRAN (R 4.1.0)
#>     scuttle              * 1.4.0      2021-10-26 [1] Bioconductor
#>     sessioninfo            1.2.1      2021-11-02 [1] CRAN (R 4.1.0)
#>  P  shape                  1.4.6      2021-05-19 [?] CRAN (R 4.1.0)
#>     shiny                  1.7.1      2021-10-02 [1] CRAN (R 4.1.0)
#>     shinyBS                0.61       2015-03-31 [1] CRAN (R 4.1.0)
#>     shinydashboard         0.7.2      2021-09-30 [1] CRAN (R 4.1.0)
#>     SingleCellExperiment * 1.16.0     2021-10-26 [1] Bioconductor
#>     sparseMatrixStats      1.6.0      2021-10-26 [1] Bioconductor
#>  P  statmod                1.4.36     2021-05-10 [?] CRAN (R 4.1.0)
#>     stringi                1.7.5      2021-10-04 [1] CRAN (R 4.1.0)
#>  P  stringr              * 1.4.0      2019-02-10 [?] CRAN (R 4.1.0)
#>     SummarizedExperiment * 1.24.0     2021-10-26 [1] Bioconductor
#>     survival               3.2-13     2021-08-24 [1] CRAN (R 4.1.0)
#>  P  svglite                2.0.0      2021-02-20 [?] CRAN (R 4.1.0)
#>     systemfonts            1.0.3      2021-10-13 [1] CRAN (R 4.1.1)
#>     tibble               * 3.1.6      2021-11-07 [1] CRAN (R 4.1.0)
#>     tidyr                * 1.1.4      2021-09-27 [1] CRAN (R 4.1.0)
#>  P  tidyselect             1.1.1      2021-04-30 [?] CRAN (R 4.1.0)
#>  P  tidyverse            * 1.3.1      2021-04-15 [?] CRAN (R 4.1.0)
#>     TMB                    1.7.22     2021-09-28 [1] CRAN (R 4.1.0)
#>     tzdb                   0.2.0      2021-10-27 [1] CRAN (R 4.1.0)
#>     UpSetR                 1.4.0      2019-05-22 [1] CRAN (R 4.1.0)
#>  P  utf8                   1.2.2      2021-07-24 [?] CRAN (R 4.1.0)
#>     variancePartition      1.24.0     2021-10-26 [1] Bioconductor
#>  P  vctrs                  0.3.8      2021-04-29 [?] CRAN (R 4.1.0)
#>  P  vipor                  0.4.5      2017-03-22 [?] CRAN (R 4.1.0)
#>     viridis                0.6.2      2021-10-13 [1] CRAN (R 4.1.0)
#>  P  viridisLite            0.4.0      2021-04-13 [?] CRAN (R 4.1.0)
#>  P  webshot                0.5.2      2019-11-22 [?] CRAN (R 4.1.0)
#>  P  withr                  2.4.2      2021-04-18 [?] CRAN (R 4.1.0)
#>     xfun                   0.28       2021-11-04 [1] CRAN (R 4.1.0)
#>     XML                    3.99-0.8   2021-09-17 [1] CRAN (R 4.1.0)
#>  P  xml2                   1.3.2      2020-04-23 [?] CRAN (R 4.1.0)
#>  P  xtable                 1.8-4      2019-04-21 [?] CRAN (R 4.1.0)
#>     XVector                0.34.0     2021-10-26 [1] Bioconductor
#>  P  yaml                   2.2.1      2020-02-01 [?] CRAN (R 4.1.0)
#>     zlibbioc               1.40.0     2021-10-26 [1] Bioconductor
#>  P  zoo                    1.8-9      2021-03-09 [?] CRAN (R 4.1.0)
#> 
#>  [1] /Users/milan/Projects/SCandwich-analyses/renv/library/R-4.1/x86_64-apple-darwin17.0
#>  [2] /Library/Frameworks/R.framework/Versions/4.1/Resources/library
#> 
#>  V ── Loaded and on-disk version mismatch.
#>  P ── Loaded and on-disk path mismatch.
#> 
#> ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────